#!/bin/sh
#
# Rerun test(s) and see if failure is reproducible
#
# Assumes you have one or more QA test failures, and hence
# the .out.bad file (this is the 1st run), and check-flakey will
# rerun the test (the 2nd run) and compare the results to see
# if this looks like a hard (deterministic) failure, or the
# result of some flakiness in the QA test.
#

tmp=/var/tmp/$$
trap "[ -f $tmp.kenj ] && ( echo "" ; cat $tmp.kenj ); rm -f $tmp.*; exit 0" 0 1 2 3 15

# This is the record that Ken uses in the QA Farm to try and track
# flakey test failures:
# <seq> <state> <failing QA hostname> <today's date>
# where the <state> is a 2 character code constructed as follows:
# - 1st char for 1st run: X (test failed) or C (callback failed) or
#   T (triaged)
# - 2nd char for 2nd run: X (test failed the same way as 1st run),
#   C (callback failed), Y (test failed differently and not C) or
#   or T (triaged) P (passed)
#
rm -f $tmp.kenj
_output_for_kenj()
{
    if [ "$USER" = "kenj" ]
    then
	# dodge numbers that look like octal!
	#
	_seq=`echo "$seq" | sed -e 's/^0*//'`
	if [ "$_seq" -gt 999 ]
	then
	    printf "%4d %s%s %s %s\n" $_seq $s1 $s2 `hostname -s` `date '+%Y-%m-%d'` >>$tmp.kenj
	else
	    printf " %03d %s%s %s %s\n" $_seq $s1 $s2 `hostname -s` `date '+%Y-%m-%d'` >>$tmp.kenj
	fi
    fi
}

# state lists (of <seq> numbers)
#
xp=""
xx=""
xc=""
xy=""
xt=""
cp=""
cx=""
cc=""
cy=""
ct=""
tp=""
tx=""	# not really possible 
ty=""
tc=""
tp=""
tt=""

# all args except -t are seq no's
#
triaged=false
list=''
while [ $# -gt 0 ]
do
    if [ "X$1" = "X-t" ]
    then
	triaged=true
    else
	list="$list $1"
    fi
    shift
done

if [ -z "$list" ]
then
    list=`echo *.out.bad \
	  | sed -e 's/.out.bad//g' \
	  | tr ' ' '\012' \
	  | sort -n \
	  | tr '\012' ' '`

    if [ "$list" = "* " ]
    then
	echo "Nothing failed, bozo!"
	exit 1
    fi
    set -- $list
fi

for seq
do
    if [ ! -f $seq.out.bad ]
    then
	echo "$seq: no .out.bad file, so nothing to compare!"
	continue
    fi
    if grep -q '^\[triaged]$' $seq.out.bad
    then
	if $triaged
	then
	    s1='T'
	else
	    echo "$seq: not rechecked [triaged]"
	    continue
	fi
    elif grep '^+++ check.callback output +++' $seq.out.bad >/dev/null
    then
	s1='C'
    else
	s1='X'
    fi
    cp $seq.out.bad $tmp.ref
    xpect=0
    if [ -f check.time ]
    then
	xpect=`sed -n <check.time -e "/^$seq /s/.* //p"`
    fi
    if [ -n "$xpect" -a "$xpect" != 0 ]
    then
	echo "$seq: checking (duration approx $xpect secs) ..."
    else
	echo "$seq: checking (duration unknown, test has never passed here?) ..."
    fi
    ./check -l $seq >/dev/null 2>&1
    if [ -f $seq.out.bad ]
    then
	# failed again
	#
	if grep '^\[triaged]$' $seq.out.bad >/dev/null
	then
	    s2='T'
	    echo "$seq: triaged"
	    [ "$s1" != 'T' ] && _output_for_kenj
	else
	    if cmp $seq.out.bad $tmp.ref >/dev/null
	    then
		s2='X'
		echo "$seq: same failure, really bad not flakey"
	    else
		if grep '^+++ check.callback output +++' $seq.out.bad >/dev/null
		then
		    s2='C'
		else
		    s2='Y'
		fi
		echo "$seq: different failure"
		_output_for_kenj
	    fi
	fi
    else
	# passed, so flakey ...
	#
	s2='P'
	echo "$seq: passed this time"
	_output_for_kenj
    fi

    case "$s1$s2"
    in
	XP)
	    if [ -z "$xp" ]
	    then
		xp="$seq"
	    else
		xp="$xp $seq"
	    fi
	    ;;
	XX)
	    if [ -z "$xx" ]
	    then
		xx="$seq"
	    else
		xx="$xx $seq"
	    fi
	    ;;
	XC)
	    if [ -z "$xc" ]
	    then
		xc="$seq"
	    else
		xc="$xc $seq"
	    fi
	    ;;
	XY)
	    if [ -z "$xy" ]
	    then
		xy="$seq"
	    else
		xy="$xy $seq"
	    fi
	    ;;
	XT)
	    if [ -z "$xt" ]
	    then
		xt="$seq"
	    else
		xt="$xt $seq"
	    fi
	    ;;
	CP)
	    if [ -z "$cp" ]
	    then
		cp="$seq"
	    else
		cp="$cp $seq"
	    fi
	    ;;
	CX)
	    if [ -z "$cx" ]
	    then
		cx="$seq"
	    else
		cx="$cx $seq"
	    fi
	    ;;
	CC)
	    if [ -z "$cc" ]
	    then
		cc="$seq"
	    else
		cc="$cc $seq"
	    fi
	    ;;
	CY)
	    if [ -z "$cy" ]
	    then
		cy="$seq"
	    else
		cy="$cy $seq"
	    fi
	    ;;
	CT)
	    if [ -z "$ct" ]
	    then
		ct="$seq"
	    else
		ct="$ct $seq"
	    fi
	    ;;
	TP)
	    if [ -z "$tp" ]
	    then
		tp="$seq"
	    else
		tp="$tp $seq"
	    fi
	    ;;
	TX)		# not really possible
	    if [ -z "$tx" ]
	    then
		tx="$seq"
	    else
		tx="$tx $seq"
	    fi
	    ;;
	TC)
	    if [ -z "$tc" ]
	    then
		tc="$seq"
	    else
		tc="$tc $seq"
	    fi
	    ;;
	TY)
	    if [ -z "$ty" ]
	    then
		ty="$seq"
	    else
		ty="$ty $seq"
	    fi
	    ;;
	TT)
	    if [ -z "$tt" ]
	    then
		tt="$seq"
	    else
		tt="$tt $seq"
	    fi
	    ;;
    esac

done

echo
echo "Summary:"
[ -n "$tt" ] && echo "Solid Triaged: $tt"
[ -n "$xx" ] && echo "Solid Failures: $xx"
[ -n "$cx" ] && echo "Solid Callback Failures: $cx"
[ -n "$xp" ] && echo "Flakey (failed then passed): $xp"
[ -n "$xc" ] && echo "Flakey (failed then callback check): $xc"
[ -n "$xy" ] && echo "Flakey (different failures): $xy"
[ -n "$xt" ] && echo "Flakey (failed then triaged): $xt"
[ -n "$cp" ] && echo "Flakey (callback check then passed): $cp"
[ -n "$cc" ] && echo "Flakey (callback check then different callback check): $cc"
[ -n "$ct" ] && echo "Flakey (callback check then triaged): $ct"
[ -n "$tp" ] && echo "Flakey (triaged then passed): $tp"
[ -n "$ty" ] && echo "Flakey (triaged then failed): $ty"
[ -n "$tc" ] && echo "Flakey (triaged then callback check): $tc"

exit
